TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import type { Metadata } from "next";2import Link from "next/link";3import { notFound } from "next/navigation";4import { ArrowLeft, ChevronDown, RefreshCw } from "lucide-react";5import { getWorkspace } from "@/lib/session";6import { internalApi, InternalApiError, type CrawlJob, type CrawlPage, type CrawlPagesPage } from "@/lib/api";7import { formatBytes, formatDate, formatMs, formatNumber, timeAgo } from "@/lib/format";8import { PageHeader } from "@/components/ui/page-header";9import { Alert } from "@/components/ui/alert";10import { Badge } from "@/components/ui/badge";11import { Button } from "@/components/ui/button";12import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";13import { CopyButton } from "@/components/ui/copy-button";14import { EmptyState } from "@/components/ui/empty-state";15import { Stat, StatGrid } from "@/components/ui/stat";16import { Table, TableBody, TableCell, TableEmpty, TableHead, TableHeader, TableRow } from "@/components/ui/table";17import { CrawlPageStatusBadge, CrawlStatusBadge } from "@/components/dashboard/crawls/crawl-status-badge";18import { CancelCrawlButton } from "@/components/dashboard/crawls/cancel-crawl-button";1920export const dynamic = "force-dynamic";21export const metadata: Metadata = { title: "Crawl" };2223const PAGE_SIZE = 100;24const PREVIEW_CHARS = 2000;25const PAGE_STATUSES = ["success", "blocked", "failed"] as const;2627type SearchParams = Promise<{ cursor?: string | string[]; status?: string | string[] }>;2829function first(v: string | string[] | undefined): string | undefined {30 return Array.isArray(v) ? v[0] : v;31}3233function Row({ label, children, mono }: { label: string; children: React.ReactNode; mono?: boolean }) {34 return (35 <div className="grid grid-cols-[140px_1fr] gap-3 px-4 py-2.5 text-[13px] sm:grid-cols-[160px_1fr]">36 <dt className="text-fg-subtle">{label}</dt>37 <dd className={`min-w-0 break-all ${mono ? "font-mono tabular" : ""}`}>{children}</dd>38 </div>39 );40}4142function optionValue(v: unknown): string {43 if (v === undefined || v === null) return "—";44 if (Array.isArray(v)) return v.length ? v.join(", ") : "—";45 if (typeof v === "object") return JSON.stringify(v);46 return String(v);47}4849const OPTION_KEYS = ["max_pages", "max_depth", "format", "same_domain", "allow_subdomains", "respect_robots", "use_sitemap", "concurrency", "delay_ms", "timeout", "main_content", "country", "network", "browser", "browser_fallback", "include_patterns", "exclude_patterns", "webhook_url"] as const;5051export default async function CrawlDetailPage({ params, searchParams }: { params: Promise<{ id: string }>; searchParams: SearchParams }) {52 const [ws, { id }, sp] = await Promise.all([getWorkspace(), params, searchParams]);53 if (!/^crawl_[A-Za-z0-9]{4,64}$/.test(id)) notFound();54 const cursor = first(sp.cursor)?.trim() || null;55 const statusFilter = first(sp.status)?.trim() || null;56 const status = statusFilter && (PAGE_STATUSES as readonly string[]).includes(statusFilter) ? statusFilter : null;5758 let job: CrawlJob | null = null;59 let pages: CrawlPagesPage = { data: [], next_cursor: null };60 let loadError: string | null = null;61 try {62 job = await internalApi.getCrawl(ws.project.id, ws.user.id, id);63 } catch (e) {64 if (e instanceof InternalApiError && (e.status === 404 || e.code === "NOT_FOUND")) notFound();65 loadError = e instanceof InternalApiError ? e.message : "The Fetcha API service is unreachable.";66 }67 if (!job) {68 return (69 <div className="flex flex-col gap-5">70 <Link href="/dashboard/crawls" className="inline-flex items-center gap-1 text-[12.5px] text-fg-muted underline-offset-4 hover:text-fg hover:underline">71 <ArrowLeft className="size-3.5" /> Crawls72 </Link>73 <Alert variant="danger" title="Could not load this crawl">74 {loadError ?? "Unknown error."}75 </Alert>76 </div>77 );78 }79 try {80 pages = await internalApi.crawlPages(ws.project.id, ws.user.id, id, { cursor, limit: PAGE_SIZE, status });81 } catch (e) {82 loadError = e instanceof InternalApiError ? e.message : "Could not load the crawled pages.";83 }8485 const active = job.status === "queued" || job.status === "running";86 const stats = job.stats ?? { discovered: 0, fetched: 0, ok: 0, blocked: 0, failed: 0, bytes: 0 };87 const hrefFor = (next: { cursor?: string | null; status?: string | null }) => {88 const q = new URLSearchParams();89 const st = next.status === undefined ? status : next.status;90 if (st) q.set("status", st);91 if (next.cursor) q.set("cursor", next.cursor);92 const s = q.toString();93 return `/dashboard/crawls/${job!.id}${s ? `?${s}` : ""}`;94 };95 // Only shown for finished jobs (a live elapsed counter would need a client component).96 const durationMs = job.started_at && job.completed_at ? new Date(job.completed_at).getTime() - new Date(job.started_at).getTime() : null;9798 return (99 <div className="flex flex-col gap-5">100 <Link href="/dashboard/crawls" className="inline-flex items-center gap-1 text-[12.5px] text-fg-muted underline-offset-4 hover:text-fg hover:underline">101 <ArrowLeft className="size-3.5" /> Crawls102 </Link>103 <PageHeader104 eyebrow="Crawl"105 title={106 <span className="inline-flex flex-wrap items-center gap-2">107 <span className="min-w-0 break-all">{job.label ?? job.domain ?? job.seed_url}</span>108 <CrawlStatusBadge status={job.status} />109 </span>110 }111 description={112 <span className="flex flex-wrap items-center gap-x-2 gap-y-1">113 <span className="inline-flex items-center gap-1 font-mono text-[12.5px] text-fg">114 {job.id} <CopyButton value={job.id} className="size-6" />115 </span>116 <span className="text-fg-subtle">·</span>117 <a href={job.seed_url} target="_blank" rel="noreferrer noopener" className="min-w-0 break-all font-mono text-[12.5px] text-accent underline-offset-4 hover:underline">118 {job.seed_url}119 </a>120 <span className="text-fg-subtle">·</span>121 <span title={formatDate(job.created_at, { timeStyle: "medium" })}>created {timeAgo(job.created_at)}</span>122 </span>123 }124 actions={125 <>126 {active ? (127 <Button variant="outline" size="sm" asChild>128 <Link href={hrefFor({ cursor })} prefetch={false}>129 <RefreshCw className="size-3.5" /> Refresh130 </Link>131 </Button>132 ) : null}133 {active ? <CancelCrawlButton id={job.id} /> : null}134 </>135 }136 />137138 {job.error ? (139 <Alert variant="danger" title={job.error.code}>140 {job.error.message}141 </Alert>142 ) : null}143 {active ? (144 <Alert variant="info" title={job.status === "queued" ? "Queued" : "Running"}>145 {job.status === "queued" ? "The job is waiting for a worker slot. " : "Pages are being fetched. "}146 This page does not update on its own; use Refresh to see progress.147 </Alert>148 ) : null}149150 <StatGrid cols={6}>151 <Stat label="Discovered" value={formatNumber(stats.discovered)} />152 <Stat label="Fetched" value={formatNumber(stats.fetched)} hint={`of ${formatNumber(Number(job.options?.max_pages ?? 0)) || "—"} max`} />153 <Stat label="OK" value={formatNumber(stats.ok)} />154 <Stat label="Blocked" value={formatNumber(stats.blocked)} />155 <Stat label="Failed" value={formatNumber(stats.failed)} />156 <Stat label="Bytes" value={formatBytes(stats.bytes)} hint={durationMs !== null ? `in ${formatMs(durationMs)}` : undefined} />157 </StatGrid>158159 <div className="grid gap-5 lg:grid-cols-[minmax(0,1fr)_320px]">160 <div className="flex min-w-0 flex-col gap-3">161 <div className="flex flex-wrap items-center justify-between gap-2">162 <h2 className="text-[13px] font-semibold uppercase tracking-wide text-fg-subtle">Pages</h2>163 <nav className="flex items-center gap-1 text-[12.5px]" aria-label="Filter pages by status">164 <FilterLink href={hrefFor({ cursor: null, status: null })} active={!status}>165 All166 </FilterLink>167 {PAGE_STATUSES.map((s) => (168 <FilterLink key={s} href={hrefFor({ cursor: null, status: s })} active={status === s}>169 {s}170 </FilterLink>171 ))}172 </nav>173 </div>174 {loadError ? (175 <Alert variant="danger" title="Could not load pages">176 {loadError}177 </Alert>178 ) : null}179 <Card className="overflow-hidden">180 <Table>181 <TableHeader>182 <TableRow className="hover:bg-transparent">183 <TableHead>URL</TableHead>184 <TableHead>Status</TableHead>185 <TableHead className="text-right">HTTP</TableHead>186 <TableHead className="text-right">Depth</TableHead>187 <TableHead className="text-right">Bytes</TableHead>188 <TableHead className="text-right">Duration</TableHead>189 <TableHead>Mode</TableHead>190 </TableRow>191 </TableHeader>192 <TableBody>193 {pages.data.length === 0 ? (194 <TableEmpty colSpan={7}>{active ? "No pages fetched yet." : status ? `No ${status} pages.` : "No pages were fetched."}</TableEmpty>195 ) : (196 pages.data.map((p) => <PageRow key={p.id} page={p} />)197 )}198 </TableBody>199 </Table>200 </Card>201 <div className="flex flex-wrap items-center justify-between gap-3 text-[12.5px] text-fg-muted">202 <span className="font-mono tabular">203 {formatNumber(pages.data.length)} page{pages.data.length === 1 ? "" : "s"} shown{cursor ? " (continued)" : ""}204 </span>205 <div className="flex items-center gap-2">206 {cursor ? (207 <Button variant="ghost" size="sm" asChild>208 <Link href={hrefFor({ cursor: null })}>First page</Link>209 </Button>210 ) : null}211 {pages.next_cursor ? (212 <Button variant="outline" size="sm" asChild>213 <Link href={hrefFor({ cursor: pages.next_cursor })} rel="next" prefetch={false}>214 Load more <ChevronDown className="size-3.5" />215 </Link>216 </Button>217 ) : null}218 </div>219 </div>220 </div>221222 <aside className="flex flex-col gap-4">223 <Card>224 <CardHeader className="pb-1">225 <CardTitle className="text-[14px]">Job</CardTitle>226 </CardHeader>227 <dl className="divide-y divide-border">228 <Row label="Domain" mono>229 {job.domain}230 </Row>231 <Row label="Created">{formatDate(job.created_at, { timeStyle: "medium" })}</Row>232 <Row label="Started">{job.started_at ? formatDate(job.started_at, { timeStyle: "medium" }) : <span className="text-fg-subtle">not yet</span>}</Row>233 <Row label="Completed">{job.completed_at ? formatDate(job.completed_at, { timeStyle: "medium" }) : <span className="text-fg-subtle">{active ? "in progress" : "—"}</span>}</Row>234 </dl>235 </Card>236 <Card>237 <CardHeader className="pb-1">238 <CardTitle className="text-[14px]">Options</CardTitle>239 </CardHeader>240 <dl className="divide-y divide-border">241 {OPTION_KEYS.filter((k) => job!.options && job!.options[k] !== undefined && job!.options[k] !== null && !(Array.isArray(job!.options[k]) && (job!.options[k] as unknown[]).length === 0)).map((k) => (242 <Row key={k} label={k} mono>243 {optionValue(job!.options[k])}244 </Row>245 ))}246 </dl>247 </Card>248 <Card>249 <CardHeader className="pb-2">250 <CardTitle className="text-[14px]">Reading this page</CardTitle>251 </CardHeader>252 <CardContent className="space-y-2 text-[12.5px] text-fg-muted">253 <p>254 <strong className="text-fg">Mode</strong> shows whether a page was fetched over plain HTTP or rendered in the managed browser after a JavaScript challenge.255 </p>256 <p>257 Click a URL row to expand the first {formatNumber(PREVIEW_CHARS)} characters of the stored content. Retrieve the full content with <code className="font-mono">GET /v1/crawl/:id/pages</code>.258 </p>259 <p>260 Each page is also a request in <Link href="/dashboard/requests" className="underline-offset-4 hover:underline">Requests</Link> with source <code className="font-mono">crawl</code>.261 </p>262 </CardContent>263 </Card>264 </aside>265 </div>266 {!pages.data.length && !active && !loadError && !status ? <EmptyState compact title="Nothing to show" description="The crawl finished without fetching a page. Check the seed URL, robots.txt and the include/exclude patterns." /> : null}267 </div>268 );269}270271function FilterLink({ href, active, children }: { href: string; active: boolean; children: React.ReactNode }) {272 return (273 <Link href={href} prefetch={false} className={`rounded-md px-2 py-1 capitalize transition-colors ${active ? "bg-bg-muted font-medium text-fg" : "text-fg-muted hover:bg-bg-subtle hover:text-fg"}`} aria-current={active ? "page" : undefined}>274 {children}275 </Link>276 );277}278279function PageRow({ page: p }: { page: CrawlPage }) {280 const preview = typeof p.content === "string" && p.content.length ? p.content.slice(0, PREVIEW_CHARS) : null;281 const truncated = typeof p.content === "string" && p.content.length > PREVIEW_CHARS;282 return (283 <TableRow className="group">284 <TableCell colSpan={7} className="p-0">285 <details className="[&_summary::-webkit-details-marker]:hidden">286 <summary className="grid cursor-pointer list-none grid-cols-[minmax(0,1fr)_7rem_4rem_4rem_5.5rem_5.5rem_5rem] items-center gap-3 px-4 py-2.5 text-[13px] hover:bg-bg-subtle/60">287 <span className="min-w-0">288 <span className="flex items-center gap-1.5">289 <ChevronDown className="size-3.5 shrink-0 text-fg-subtle transition-transform group-has-[details[open]]:rotate-180" aria-hidden />290 <span className="truncate font-mono text-[12.5px]" title={p.url}>291 {p.url}292 </span>293 </span>294 {p.title ? (295 <span className="block truncate pl-5 text-[12px] text-fg-muted" title={p.title}>296 {p.title}297 </span>298 ) : null}299 {p.error_code ? <span className="block pl-5 font-mono text-[11.5px] text-danger">{p.error_code}</span> : null}300 </span>301 <span>302 <CrawlPageStatusBadge status={p.status} />303 </span>304 <span className="text-right font-mono tabular">{p.http_status ?? <span className="text-fg-subtle">—</span>}</span>305 <span className="text-right font-mono tabular text-fg-muted">{p.depth}</span>306 <span className="text-right font-mono tabular text-fg-muted">{formatBytes(p.bytes)}</span>307 <span className="text-right font-mono tabular text-fg-muted">{formatMs(p.duration_ms)}</span>308 <span>{p.mode ? <Badge variant={p.mode === "browser" ? "accent" : "outline"}>{p.mode === "browser" ? "Browser" : "HTTP"}</Badge> : <span className="text-fg-subtle">—</span>}</span>309 </summary>310 <div className="border-t border-border bg-bg-subtle/40 px-4 py-3">311 <dl className="mb-3 grid gap-x-6 gap-y-1 text-[12px] text-fg-muted sm:grid-cols-[auto_minmax(0,1fr)]">312 {p.final_url && p.final_url !== p.url ? (313 <>314 <dt className="text-fg-subtle">Final URL</dt>315 <dd className="min-w-0 break-all font-mono">{p.final_url}</dd>316 </>317 ) : null}318 {p.description ? (319 <>320 <dt className="text-fg-subtle">Description</dt>321 <dd className="min-w-0 break-words">{p.description}</dd>322 </>323 ) : null}324 <dt className="text-fg-subtle">Content type</dt>325 <dd className="font-mono">{p.content_type ?? "—"}</dd>326 <dt className="text-fg-subtle">Links</dt>327 <dd className="font-mono tabular">{p.links_count ?? "—"}</dd>328 <dt className="text-fg-subtle">Fetched</dt>329 <dd>{p.fetched_at ? formatDate(p.fetched_at, { timeStyle: "medium" }) : "—"}</dd>330 </dl>331 {preview ? (332 <>333 <pre className="max-h-[360px] overflow-auto whitespace-pre-wrap break-words rounded-md border border-border bg-bg p-3 font-mono text-[12px] leading-relaxed scrollbar-thin">{preview}</pre>334 {truncated ? <p className="mt-1.5 text-[11.5px] text-fg-subtle">Showing the first {formatNumber(PREVIEW_CHARS)} of {formatNumber(p.content!.length)} characters.</p> : null}335 </>336 ) : (337 <p className="text-[12px] text-fg-subtle">No stored content for this page.</p>338 )}339 </div>340 </details>341 </TableCell>342 </TableRow>343 );344}345